import requests
import json
import pandas as pd
from quant_researcher.quant.project_tool.celebrity import union_filter
from utils import df_into_db
import ast

api_key = 'b952a5c5-dbd9-4671-8a69-bb5949aa8190'


def get_all_coinlist_from_coinmarketcap(status='inactive,active,untracked', method="http", tag="all"):
    """
    获取coinmarketcap网站上目前支持的所有coins

    参考：https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyMap

    :param status: "active" Only active cryptocurrencies are returned by default.
                   Pass inactive to get a list of cryptocurrencies that are no longer active.
                   Pass untracked to get a list of cryptocurrencies that are listed but do not yet meet methodology requirements to have tracked markets available.
                   You may pass one or more comma-separated values.
    :param str method: 数据获取的方法， 支持'http' 网页爬虫, 'api'官方api, 通过api的话需要指定api_key
    :param str tag: 某类币，支持'all', 'stablecoin', 'asset-backed-stablecoin', 'defi', 'defi-2', 'a16z-portfolio', 'binance-launchpad', 'memes', 'sports', 'web3',
                            'decentralized-exchange', 'pow', 等等，所有支持的可以通过以下函数计算得到的all_df的tags列所支持的所有选项
    :param kwargs: 关键字参数
    :return:
        输出的列包含 id, name, symbol, slug, rank, is_active, first_histotrical_data, platform, status
    """


    if method == 'http':
        header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
    elif method == 'api':
        header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
                  'X-CMC_PRO_API_KEY': api_key}
    else:
        raise NotImplementedError

    all_df_list = []
    flag = True
    i = 1
    start = 1
    page_size = 5000
    while flag and (start < 50000):
        print(f'开始进行获取第{i}页数据')
        if method == 'http':
            # 下面这个url只能获取active的
            # url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listing?start=1&limit=20000&sortBy=market_cap&sortType=desc&convert=USD&' \
            #       f'cryptoType=all&tagType=all&audited=false&' \
            #       f'aux=tags,ath,atl,high24h,low24h,num_market_pairs,cmc_rank,date_added,max_supply,circulating_supply,total_supply,volume_7d,volume_30d,self_reported_circulating_supply,self_reported_market_cap'
            # 下面这个url可以获取inactive,active,untracked
            url = f'https://api.coinmarketcap.com/data-api/v3/map/all?listing_status={status}&' \
                  f'exchangeAux=is_active,status&' \
                  f'cryptoAux=is_active,status,platform,first_historical_data,last_historical_data&' \
                  f'start={start}&limit={page_size}'
            a = requests.get(url, headers=header, timeout=30)
            res = json.loads(a.text)
            df = pd.DataFrame(res['data']['cryptoCurrencyMap'])
        elif method == 'api':
            url = f'https://pro-api.coinmarketcap.com/v1/cryptocurrency/map'
            params = {"listing_status": status,
                      "start": start,
                      "limit": page_size,
                      "sort": "id",
                      'aux': "platform,first_historical_data,last_historical_data,is_active,status",
                      }
            a = requests.get(url, params=params, headers=header, timeout=30)
            res = json.loads(a.text)
            df = pd.DataFrame(res['data'])
        else:
            raise NotImplementedError

        all_df_list.append(df)
        if df.shape[0] < page_size:  # 数据已经全部获取，无需再获取
            flag = False
        else:
            start = start + page_size
            i = i + 1

    all_df = pd.concat(all_df_list, axis=0, ignore_index=True)
    all_df[['first_historical_data', 'last_historical_data', 'platform']] = all_df[['first_historical_data', 'last_historical_data', 'platform']].fillna('')
    all_df['rank'] = all_df['rank'].fillna(0)
    all_df['platform'] = all_df['platform'].apply(lambda x: json.dumps(x))

    # 获取每个币的标签信息
    url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listing?start=1&limit=20000&sortBy=market_cap&sortType=desc&convert=USD&' \
          f'cryptoType=all&tagType=all&audited=false&' \
          f'aux=tags'
    a = requests.get(url, headers=header, timeout=30)
    res = json.loads(a.text)
    df = pd.DataFrame(res['data']['cryptoCurrencyList'])

    # 币的信息和标签信息合并
    all_df = all_df.merge(df[['id', 'tags']], how='left', on=['id'])
    all_df['tags'] = all_df['tags'].fillna('')
    all_df['tags'] = all_df['tags'].apply(lambda x: json.dumps(x))
    df_into_db(all_df, db_name = "all_history_ohlcvm_coinmarketcap", table_name = "all_coin_list_coinmarketcap")
    # 所有支持的tags
    all_tags_list = []
    index_num_list = []
    for i, this_tags in enumerate(all_df['tags']):
        if this_tags == []:
            pass
        elif str(this_tags) == 'nan':
            pass
        else:
            all_tags_list.append(this_tags)
            if tag == 'all':
                index_num_list = list(all_df.index)
            else:
                if tag.lower() in this_tags:
                    index_num_list.append(i)
    all_tags_list = [ast.literal_eval(json.loads(x)) for x in all_tags_list]
    all_tags_list = union_filter(all_tags_list, how_to_deal='or')
    all_tags_list.sort()
    all_tags = pd.Series(all_tags_list)
    all_tags = all_tags.to_frame()
    all_tags.columns = ["tag"]
    df_into_db(all_tags, db_name = "all_history_ohlcvm_coinmarketcap", table_name = "all_coin_tag_list_coinmarketcap")









if __name__ == '__main__':
    get_all_coinlist_from_coinmarketcap()